import pandas as pd
import numpy as np
import scipy.io
import re
import sys
import warnings
import pickle

import pickle
from IPython.core.display import display, HTML, Markdown

/tmp/ipykernel_21689/659417397.py:2: DeprecationWarning: Importing display from IPython.core.display is deprecated since IPython 7.14, please import from IPython display
  from IPython.core.display import display, HTML, Markdown

from mvpa2.datasets.mri import fmri_dataset

Failed to import duecredit due to No module named 'duecredit'
/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/mvpa2/datasets/base.py:465: DeprecationWarning: `np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.
Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations
  def __init__(self, shape=None, sid=None, fid=None, dtype=np.float):

class BehavioralDataNotFoundForBrainDataException(Exception):
    """Behavioral data could not be matched to a subject."""
    pass

from nilearn.masking import unmask

Replicating earlier work on mvpa. Try not to overly complicate it--the main point is just to verify we get similar results on a different package to validate prior work. ANd we are primarily interested in validating the very high cross-validation results I got with nltools. Should aim for readable code.

Version 5 uses scikit-learn directly, bypassing mvpa2's framework altogether. We also implement a 'forced choice' scorer.

from visualization import visualize_corr
import mvpa2

/gpfs/projects/sanlab/shared/DEV/DEV_scripts/fMRI/ml/SST/visualization.py:7: DeprecationWarning: Importing display from IPython.core.display is deprecated since IPython 7.14, please import from IPython display
  from IPython.core.display import display, HTML, Markdown

import sys
import os

sys.path.append(os.path.abspath("../../ml/"))

Load

nonbids_data_path = "/gpfs/projects/sanlab/shared/DEV/nonbids_data/"
ml_data_folderpath = "/gpfs/projects/sanlab/shared/DEV/nonbids_data/fMRI/ml"

include_exclude_list = pd.read_csv("../nsc_subject_exclusions.csv")

test_train_df_raw = pd.read_csv(nonbids_data_path + "fMRI/ml/train_test_markers_20211027T173724.csv")
test_train_df_raw = test_train_df_raw.merge(include_exclude_list[include_exclude_list.Task=='SST'],left_on='sub_label',right_on='SubjectId',how='left')
test_train_df_raw.loc[test_train_df_raw.Include.isna(),'Include'] = True
test_train_df = test_train_df_raw[test_train_df_raw.Include==True]
exclude_subjects = ['DEV061','DEV185','DEV187','DEV189','DEV190','DEV192','DEV198','DEV203','DEV220','DEV221']
train_subjs = test_train_df.loc[test_train_df.SplitGroup=='Train','sub_label'].tolist()#only get the train subjects; ignore those previously marked hold-out

train_subjs_selected = [ts for ts in train_subjs if (ts not in exclude_subjects)]

individual_differences = pd.read_csv(ml_data_folderpath + "/data_by_ppt_2022_02_26.csv")
individual_differences = individual_differences.rename(columns={'SID':'subject'})
individual_differences['wave']=1

We probably actually want to start the pipeline from the betas rather than loading from pickle. to be continued...

from mvpa_pipeline_utils import get_Brain_Data_betas_as_mvpa_for_sub, import_beta_series_pymvpa2, sa_to_df

new code

from mvpa_sst_utils import *

/gpfs/projects/sanlab/shared/DEV/DEV_scripts/fMRI/ml/SST/mvpa_sst_utils.py:10: DeprecationWarning: Importing display from IPython.core.display is deprecated since IPython 7.14, please import from IPython display
  from IPython.core.display import display, HTML, Markdown
/gpfs/projects/sanlab/shared/DEV/DEV_scripts/fMRI/ml/SST/outlier_detection.py:7: DeprecationWarning: Importing display from IPython.core.display is deprecated since IPython 7.14, please import from IPython display
  from IPython.core.display import display, HTML, Markdown

from sklearn.model_selection import LeaveOneGroupOut

def remove_selected_outliers_mvpa_tesq_study(
    ind_div_combined,
    show_plot=False,
    neural_measures = ['CorGo_ln_inv_pred_prob_CorrectGo','CorStop_ln_inv_pred_prob_CorrectStop']):
    idc_outliers_removed = remove_selected_outliers(ind_div_combined,
    ['BFI_extraversion','RMQ_locomotion','ses_aggregate','PLAN_cognitive_strategies',
     'SST_SSRT','BIS_11','BSCS','TESQ_E_suppression', 'TESQ_E_avoidance_of_temptations', 
     'TESQ_E_goal_deliberation', 'TESQ_E_controlling_temptations', 'TESQ_E_distraction',
     'TESQ_E_goal_and_rule_setting','TESQ_E_sum','EDM','RS','TRSQ','ROC_Crave_Regulate_Minus_Look',
     'SRHI_unhealthy','cancer_promoting_minus_preventing_FFQ','bf_1'] + neural_measures,
    show_plot=show_plot)
    return(idc_outliers_removed)

from outlier_detection import *

from sklearn.svm import SVC
#from mvpa2.measures.base import CrossValidation
#from mvpa2.clfs.meta import NFoldPartitioner
#from mvpa2.clfs.svm import LinearCSVMC
from sklearn.calibration import CalibratedClassifierCV

from sklearn.model_selection import LeaveOneGroupOut

from generic_utils import *

whole brain

Now let's scale that up to the full dataset.

masks

We get the same file this time, but we apply a mask before doing learning.

## additional masks

all_masks = get_all_masks(ml_data_folderpath)

for mask_name in all_masks['mask_name']:
#    mask_value = all_masks[mask_name]
    print(mask_name,end=", ")

planning_association-test_z_FDR_0.01, executive function_association-test_z_FDR_0.01, response inhibition_association-test_z_FDR_0.01, inhibition_association-test_z_FDR_0.01, error_association-test_z_FDR_0.01, harvardoxford-cortical_prob_Frontal Orbital Cortex, harvardoxford-cortical_prob_Inferior Frontal Gyrus, pars opercularis, harvardoxford-cortical_prob_Inferior Frontal Gyrus, pars triangularis, 

Next to do:

1. Consider re-implementing LinearSVM and getting similarity rather than this probability measure
2. Visualize for all the other values.

Now we've decoupled the learner from the framework where we apply it, we can pass in any arbitrary learner. It just has to output:

1. predictions
2. prediction confidence
3. the classifier itself

Re-implementing v3

This is a useful sanity check, becuase it's helpufl to try and explain why we're doing so much better than rpeviously.

brain_data_filepath = (
    "/gpfs/projects/sanlab/shared/DEV/nonbids_data/fMRI/ml/SST/mvpa_Dataset_conditions_84subs_correct_cond.pkl"
)

with open(brain_data_filepath, 'rb') as pkl_file:
    Brain_Data_allsubs = pickle.load(pkl_file)

Brain_Data_allsubs = setup_metadata(Brain_Data_allsubs)

linear_results_with_similarity = do_forced_choice(
    Brain_Data_allsubs,
    get_predict_and_prob=do_LinearSVC)

......

LinearSVC_correlation_wholebrain = linear_results_with_similarity

forced_choice_results = linear_results_with_similarity

prediction = np.mean(forced_choice_results['sample_wise']['target_y']==forced_choice_results['sample_wise']['pred_y'])
forced_choice_prediction = np.mean(forced_choice_results['sample_wise']['target_y']==forced_choice_results['sample_wise']['pred_y_forced_choice'])

(prediction, forced_choice_prediction)

(0.8518518518518519, 0.9876543209876543)

Formerly this was .85, .95, which was remarkable--we've now pretty clearly reproduced the result we were getting using nltools..

However, after I updated it based on documentation here (https://nilearn.github.io/modules/generated/nilearn.decoding.Decoder.html) it performed less well. Not sure why the classifier is performing less well in this setup than the former one - perhaps hte sort of normalization that I am doing on each side.

sample_wise_results = forced_choice_results['sample_wise']
cs_cs_prob = sample_wise_results.loc[sample_wise_results.target_y=='CorrectStop']

individual_differences['subj_wave'] = individual_differences.subject+"_"+individual_differences.wave.astype(str)
full_dataset_cs = individual_differences.merge(cs_cs_prob,how='outer',left_on='subj_wave',right_on='chunks')

from outlier_detection import *

full_dataset_cs = remove_selected_outliers_mvpa_tesq_study(
    full_dataset_cs,
    neural_measures = ['pred_prob_CorrectStop'])

visualize_corr('pred_prob_CorrectStop','bf_1',full_dataset_cs)

bf_1

r=-0.27; p-value=0.0235

rho=-0.30; p-value=0.0112

visualize_corr('pred_prob_CorrectStop','cancer_promoting_minus_preventing_FFQ',full_dataset_cs)

cancer_promoting_minus_preventing_FFQ

r=0.04; p-value=0.7335

rho=0.03; p-value=0.8176

visualize_corr('pred_prob_CorrectStop','TESQ_E_sum',full_dataset_cs)

TESQ_E_sum

r=-0.08; p-value=0.4795

rho=-0.07; p-value=0.5708

v3 mask

forced_choice_results_dict = {}
for m_i,m_r in all_masks.iterrows():
    mask_name=m_r['mask_name']
    brain_data_filepath = (
        "/gpfs/projects/sanlab/shared/DEV/nonbids_data/fMRI/ml/SST/mvpa_Dataset_conditions_84subs_correct_cond_" +
        mask_name
        + ".pkl"
    )
    if os.path.exists(brain_data_filepath):
        with open(brain_data_filepath, 'rb') as pkl_file:
            Brain_Data_allsubs = pickle.load(pkl_file)
            
        print(mask_name)

        Brain_Data_allsubs = setup_metadata(Brain_Data_allsubs)
        
        print(sum(sum(Brain_Data_allsubs.samples>0))/len(Brain_Data_allsubs.samples))
        
        masked_preview = unmask(Brain_Data_allsubs.samples[0], Brain_Data_allsubs.a['mask_nifti'].value)
        plotting.plot_stat_map(masked_preview,vmax=18)

        forced_choice_results = do_forced_choice(Brain_Data_allsubs,get_predict_and_prob=do_LinearSVC)
        print("")
        
        prediction = np.mean(forced_choice_results['sample_wise']['target_y']==forced_choice_results['sample_wise']['pred_y'])
        forced_choice_prediction = np.mean(forced_choice_results['sample_wise']['target_y']==forced_choice_results['sample_wise']['pred_y_forced_choice'])
        
        print(prediction,forced_choice_prediction)

        forced_choice_results_dict[mask_name]=forced_choice_results
        

planning_association-test_z_FDR_0.01
2705.6913580246915

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/nilearn/plotting/img_plotting.py:348: FutureWarning: Default resolution of the MNI template will change from 2mm to 1mm in version 0.10.0
  anat_img = load_mni152_template()
/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/scipy/ndimage/measurements.py:305: DeprecationWarning: In future, it will be an error for 'np.bool_' scalars to be interpreted as an index
  return _nd_image.find_objects(input, max_label)

.................................................................................
0.7407407407407407 0.9629629629629629
executive function_association-test_z_FDR_0.01
99.72839506172839

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/scipy/ndimage/measurements.py:305: DeprecationWarning: In future, it will be an error for 'np.bool_' scalars to be interpreted as an index
  return _nd_image.find_objects(input, max_label)

.................................................................................
0.7037037037037037 0.8641975308641975
response inhibition_association-test_z_FDR_0.01
1958.111111111111

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/scipy/ndimage/measurements.py:305: DeprecationWarning: In future, it will be an error for 'np.bool_' scalars to be interpreted as an index
  return _nd_image.find_objects(input, max_label)

.................................................................................
0.6851851851851852 0.9753086419753086
inhibition_association-test_z_FDR_0.01
667.4135802469136

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/scipy/ndimage/measurements.py:305: DeprecationWarning: In future, it will be an error for 'np.bool_' scalars to be interpreted as an index
  return _nd_image.find_objects(input, max_label)

.................................................................................
0.6172839506172839 0.8765432098765432
harvardoxford-cortical_prob_Frontal Orbital Cortex
3142.7654320987654

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/scipy/ndimage/measurements.py:305: DeprecationWarning: In future, it will be an error for 'np.bool_' scalars to be interpreted as an index
  return _nd_image.find_objects(input, max_label)

.................................................................................
0.6666666666666666 0.9135802469135802
harvardoxford-cortical_prob_Inferior Frontal Gyrus, pars opercularis
2416.91975308642

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/scipy/ndimage/measurements.py:305: DeprecationWarning: In future, it will be an error for 'np.bool_' scalars to be interpreted as an index
  return _nd_image.find_objects(input, max_label)

.................................................................................
0.7098765432098766 0.9382716049382716
harvardoxford-cortical_prob_Inferior Frontal Gyrus, pars triangularis
2102.1666666666665

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/scipy/ndimage/measurements.py:305: DeprecationWarning: In future, it will be an error for 'np.bool_' scalars to be interpreted as an index
  return _nd_image.find_objects(input, max_label)

.................................................................................
0.691358024691358 0.9012345679012346

LinearSVC_correlation_masked = forced_choice_results_dict

for k in forced_choice_results_dict.keys():
    v=forced_choice_results_dict[k]
    print(k)
    print_forced_choice_results(v,individual_differences,['pred_prob_CorrectStop'])

planning_association-test_z_FDR_0.01
(0.7407407407407407, 0.9629629629629629)

bf_1

r=-0.35; p-value=0.0028

rho=-0.35; p-value=0.0025

cancer_promoting_minus_preventing_FFQ

r=-0.09; p-value=0.4478

rho=-0.05; p-value=0.6894

TESQ_E_sum

r=-0.08; p-value=0.5183

rho=-0.09; p-value=0.4228

executive function_association-test_z_FDR_0.01
(0.7037037037037037, 0.8641975308641975)

bf_1

r=-0.13; p-value=0.2804

rho=-0.04; p-value=0.7465

cancer_promoting_minus_preventing_FFQ

r=-0.07; p-value=0.5597

rho=-0.07; p-value=0.5504

TESQ_E_sum

r=0.14; p-value=0.2309

rho=0.15; p-value=0.1901

response inhibition_association-test_z_FDR_0.01
(0.6851851851851852, 0.9753086419753086)

bf_1

r=-0.19; p-value=0.1176

rho=-0.22; p-value=0.0650

cancer_promoting_minus_preventing_FFQ

r=-0.01; p-value=0.9166

rho=-0.03; p-value=0.8206

TESQ_E_sum

r=-0.15; p-value=0.1952

rho=-0.15; p-value=0.2082

inhibition_association-test_z_FDR_0.01
(0.6172839506172839, 0.8765432098765432)

bf_1

r=-0.18; p-value=0.1208

rho=-0.15; p-value=0.1974

cancer_promoting_minus_preventing_FFQ

r=-0.01; p-value=0.9267

rho=-0.04; p-value=0.7113

TESQ_E_sum

r=-0.17; p-value=0.1410

rho=-0.18; p-value=0.1322

harvardoxford-cortical_prob_Frontal Orbital Cortex
(0.6666666666666666, 0.9135802469135802)

bf_1

r=-0.15; p-value=0.2124

rho=-0.14; p-value=0.2556

cancer_promoting_minus_preventing_FFQ

r=-0.16; p-value=0.1636

rho=-0.17; p-value=0.1435

TESQ_E_sum

r=0.04; p-value=0.7654

rho=0.08; p-value=0.5109

harvardoxford-cortical_prob_Inferior Frontal Gyrus, pars opercularis
(0.7098765432098766, 0.9382716049382716)

bf_1

r=0.09; p-value=0.4585

rho=0.08; p-value=0.4827

cancer_promoting_minus_preventing_FFQ

r=0.05; p-value=0.6884

rho=0.07; p-value=0.5345

TESQ_E_sum

r=0.00; p-value=0.9742

rho=-0.00; p-value=0.9861

harvardoxford-cortical_prob_Inferior Frontal Gyrus, pars triangularis
(0.691358024691358, 0.9012345679012346)

bf_1

r=0.18; p-value=0.1408

rho=0.17; p-value=0.1599

cancer_promoting_minus_preventing_FFQ

r=-0.13; p-value=0.2441

rho=-0.09; p-value=0.4030

TESQ_E_sum

r=0.15; p-value=0.2096

rho=0.13; p-value=0.2717

Apply the main analysis, looping through masks

forced_choice_results_dict = {}
for m_i,m_r in all_masks.iterrows():
    mask_name=m_r['mask_name']
    brain_data_filepath = (
        "/gpfs/projects/sanlab/shared/DEV/nonbids_data/fMRI/ml/SST/mvpa_Dataset_conditions_84subs_correct_cond_" +
        mask_name
        + ".pkl"
    )
    if os.path.exists(brain_data_filepath):
        with open(brain_data_filepath, 'rb') as pkl_file:
            Brain_Data_allsubs = pickle.load(pkl_file)
            
        print(mask_name)

        Brain_Data_allsubs = setup_metadata(Brain_Data_allsubs)
        
        print(sum(sum(Brain_Data_allsubs.samples>0))/len(Brain_Data_allsubs.samples))
        
        masked_preview = unmask(Brain_Data_allsubs.samples[0], Brain_Data_allsubs.a['mask_nifti'].value)
        plotting.plot_stat_map(masked_preview,vmax=18)
        plotting.show()
        forced_choice_results = do_forced_choice(Brain_Data_allsubs,get_predict_and_prob=do_SVC)
        print("")
        
        prediction = np.mean(forced_choice_results['sample_wise']['target_y']==forced_choice_results['sample_wise']['pred_y'])
        forced_choice_prediction = np.mean(forced_choice_results['sample_wise']['target_y']==forced_choice_results['sample_wise']['pred_y_forced_choice'])
        
        forced_choice_results['sample_wise']['ln_inv_pred_prob_CorrectGo']=inv_log_transform(forced_choice_results['sample_wise']['pred_prob_CorrectGo'])
        forced_choice_results['sample_wise']['ln_inv_pred_prob_CorrectStop']=inv_log_transform(forced_choice_results['sample_wise']['pred_prob_CorrectStop'])


        
        print(prediction,forced_choice_prediction)

        forced_choice_results_dict[mask_name]=forced_choice_results
        print_forced_choice_results(forced_choice_results,individual_differences,['ln_inv_pred_prob_CorrectStop'])

planning_association-test_z_FDR_0.01
2705.6913580246915

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/scipy/ndimage/measurements.py:305: DeprecationWarning: In future, it will be an error for 'np.bool_' scalars to be interpreted as an index
  return _nd_image.find_objects(input, max_label)

.................................................................................
0.9320987654320988 0.9876543209876543
(0.9320987654320988, 0.9876543209876543)

bf_1

r=0.29; p-value=0.0129

rho=0.31; p-value=0.0080

cancer_promoting_minus_preventing_FFQ

r=0.08; p-value=0.4800

rho=0.13; p-value=0.2523

TESQ_E_sum

r=-0.05; p-value=0.6948

rho=-0.14; p-value=0.2391

executive function_association-test_z_FDR_0.01
99.72839506172839

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/scipy/ndimage/measurements.py:305: DeprecationWarning: In future, it will be an error for 'np.bool_' scalars to be interpreted as an index
  return _nd_image.find_objects(input, max_label)

.................................................................................
0.9135802469135802 0.9753086419753086
(0.9135802469135802, 0.9753086419753086)

bf_1

r=0.31; p-value=0.0087

rho=0.34; p-value=0.0036

cancer_promoting_minus_preventing_FFQ

r=0.05; p-value=0.6510

rho=0.06; p-value=0.5762

TESQ_E_sum

r=-0.08; p-value=0.4829

rho=-0.12; p-value=0.2936

response inhibition_association-test_z_FDR_0.01
1958.111111111111

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/scipy/ndimage/measurements.py:305: DeprecationWarning: In future, it will be an error for 'np.bool_' scalars to be interpreted as an index
  return _nd_image.find_objects(input, max_label)

.................................................................................
0.9382716049382716 0.9753086419753086
(0.9382716049382716, 0.9753086419753086)

bf_1

r=0.20; p-value=0.0966

rho=0.20; p-value=0.0876

cancer_promoting_minus_preventing_FFQ

r=0.13; p-value=0.2391

rho=0.24; p-value=0.0324

TESQ_E_sum

r=-0.07; p-value=0.5423

rho=-0.12; p-value=0.2958

inhibition_association-test_z_FDR_0.01
667.4135802469136

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/scipy/ndimage/measurements.py:305: DeprecationWarning: In future, it will be an error for 'np.bool_' scalars to be interpreted as an index
  return _nd_image.find_objects(input, max_label)

.................................................................................
0.9259259259259259 0.9753086419753086
(0.9259259259259259, 0.9753086419753086)

bf_1

r=0.21; p-value=0.0819

rho=0.19; p-value=0.1145

cancer_promoting_minus_preventing_FFQ

r=0.18; p-value=0.1024

rho=0.29; p-value=0.0098

TESQ_E_sum

r=-0.11; p-value=0.3455

rho=-0.18; p-value=0.1323

harvardoxford-cortical_prob_Frontal Orbital Cortex
3142.7654320987654

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/scipy/ndimage/measurements.py:305: DeprecationWarning: In future, it will be an error for 'np.bool_' scalars to be interpreted as an index
  return _nd_image.find_objects(input, max_label)

.................................................................................
0.9506172839506173 0.9753086419753086
(0.9506172839506173, 0.9753086419753086)

bf_1

r=0.23; p-value=0.0473

rho=0.23; p-value=0.0537

cancer_promoting_minus_preventing_FFQ

r=0.17; p-value=0.1273

rho=0.23; p-value=0.0431

TESQ_E_sum

r=-0.19; p-value=0.1046

rho=-0.28; p-value=0.0166

harvardoxford-cortical_prob_Inferior Frontal Gyrus, pars opercularis
2416.91975308642

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/scipy/ndimage/measurements.py:305: DeprecationWarning: In future, it will be an error for 'np.bool_' scalars to be interpreted as an index
  return _nd_image.find_objects(input, max_label)

.................................................................................
0.9320987654320988 0.9753086419753086
(0.9320987654320988, 0.9753086419753086)

bf_1

r=0.06; p-value=0.6208

rho=0.08; p-value=0.4845

cancer_promoting_minus_preventing_FFQ

r=0.04; p-value=0.7496

rho=0.09; p-value=0.4389

TESQ_E_sum

r=-0.08; p-value=0.4872

rho=-0.17; p-value=0.1371

harvardoxford-cortical_prob_Inferior Frontal Gyrus, pars triangularis
2102.1666666666665

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/scipy/ndimage/measurements.py:305: DeprecationWarning: In future, it will be an error for 'np.bool_' scalars to be interpreted as an index
  return _nd_image.find_objects(input, max_label)

.................................................................................
0.9444444444444444 0.9753086419753086
(0.9444444444444444, 0.9753086419753086)

bf_1

r=0.18; p-value=0.1222

rho=0.19; p-value=0.1184

cancer_promoting_minus_preventing_FFQ

r=0.19; p-value=0.0943

rho=0.26; p-value=0.0205

TESQ_E_sum

r=0.02; p-value=0.8863

rho=-0.07; p-value=0.5387

SVC_proba_masked = forced_choice_results_dict

print("whole brain")
brain_data_filepath = (
    "/gpfs/projects/sanlab/shared/DEV/nonbids_data/fMRI/ml/SST/mvpa_Dataset_conditions_84subs_correct_cond.pkl"
)

with open(brain_data_filepath, 'rb') as pkl_file:
    Brain_Data_allsubs = pickle.load(pkl_file)

Brain_Data_allsubs = setup_metadata(Brain_Data_allsubs)

print(sum(sum(Brain_Data_allsubs.samples>0))/len(Brain_Data_allsubs.samples))

#masked_preview = unmask(Brain_Data_allsubs.samples[0], Brain_Data_allsubs.a['mask_nifti'].value)
#plotting.plot_stat_map(masked_preview,vmax=18)

forced_choice_results = do_forced_choice(Brain_Data_allsubs,get_predict_and_prob=do_SVC)


sample_wise_results = forced_choice_results['sample_wise']
prediction = np.mean(forced_choice_results['sample_wise']['target_y']==forced_choice_results['sample_wise']['pred_y'])
forced_choice_prediction = np.mean(forced_choice_results['sample_wise']['target_y']==forced_choice_results['sample_wise']['pred_y_forced_choice'])

forced_choice_results['sample_wise']['ln_inv_pred_prob_CorrectGo']=inv_log_transform(sample_wise_results['pred_prob_CorrectGo'])
forced_choice_results['sample_wise']['ln_inv_pred_prob_CorrectStop']=inv_log_transform(sample_wise_results['pred_prob_CorrectStop'])



print(prediction,forced_choice_prediction)

forced_choice_results_dict[mask_name]=forced_choice_results
print_forced_choice_results(forced_choice_results,individual_differences,['ln_inv_pred_prob_CorrectStop'])

whole brain
127427.37037037036
.................................................................................0.9506172839506173 0.9876543209876543
(0.9506172839506173, 0.9876543209876543)

bf_1

r=0.23; p-value=0.0572

rho=0.25; p-value=0.0359

cancer_promoting_minus_preventing_FFQ

r=0.20; p-value=0.0721

rho=0.16; p-value=0.1579

TESQ_E_sum

r=-0.16; p-value=0.1745

rho=-0.22; p-value=0.0628

SVC_proba_wholebrain = forced_choice_results

Contrast between CorrectStop and CorrectGo...

def print_regression_results(forced_choice_results, neural_measures = ['mae']):
    sample_wise_results = forced_choice_results['sample_wise']
#    cs_cs_prob = sample_wise_results.loc[sample_wise_results.target_y=='CorrectStop']

    individual_differences['subj_wave'] = individual_differences.subject+"_"+individual_differences.wave.astype(str)
    full_dataset_cs = individual_differences.merge(sample_wise_results,how='outer',left_on='subj_wave',right_on='chunks')
    print(full_dataset_cs.columns)
    full_dataset_cs = remove_selected_outliers_mvpa_tesq_study(
        full_dataset_cs,neural_measures = neural_measures,
        show_plot=False)
    
    #https://docs.google.com/presentation/d/10tKHw1VF2WiMapvKXkh2C9VeFHk6iCjTmc0tDTiugOk/edit#slide=id.g11512b79600_0_10
    for nm in neural_measures:
        visualize_corr(nm,'bf_1',full_dataset_cs)
        visualize_corr(nm,'cancer_promoting_minus_preventing_FFQ',full_dataset_cs)
        visualize_corr(nm,'TESQ_E_sum',full_dataset_cs)
    
    

    

forced_choice_results_dict = {}
for m_i,m_r in all_masks.iterrows():
    mask_name=m_r['mask_name']
    brain_data_filepath = (
        "/gpfs/projects/sanlab/shared/DEV/nonbids_data/fMRI/ml/SST/mvpa_Dataset_conditions_84subs_correct_stop_go_contrast_" +
        mask_name
        + ".pkl"
    )
    if os.path.exists(brain_data_filepath):

        target='SST_SSRT'

        with open(brain_data_filepath, 'rb') as pkl_file:
            Brain_Data_allsubs = pickle.load(pkl_file)

        Brain_Data_allsubs = Brain_Data_allsubs[np.isnan(Brain_Data_allsubs.sa[target].value)==False]

        print(mask_name)

        Brain_Data_allsubs = setup_metadata(Brain_Data_allsubs,target,standardize=True)

        print(sum(sum(Brain_Data_allsubs.samples>0))/len(Brain_Data_allsubs.samples))

        masked_preview = unmask(Brain_Data_allsubs.samples[0], Brain_Data_allsubs.a['mask_nifti'].value)
        plotting.plot_stat_map(masked_preview,vmax=18)
        plotting.show()
        forced_choice_results = do_regression(Brain_Data_allsubs,get_predict_and_prob=do_Ridge)
        print("")


        forced_choice_results_dict[mask_name]=forced_choice_results
        print_regression_results(forced_choice_results,['mae'])
        
mae_predictivity_masked = forced_choice_results_dict

planning_association-test_z_FDR_0.01
1685.1518987341772

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/scipy/ndimage/measurements.py:305: DeprecationWarning: In future, it will be an error for 'np.bool_' scalars to be interpreted as an index
  return _nd_image.find_objects(input, max_label)

...............................................................................
Index(['subject', 'cancer_promoting_minus_preventing_FCI',
       'cancer_promoting_minus_preventing_FFQ',
       'cancer_promoting_minus_preventing_craved_FCI',
       'cancer_promoting_minus_preventing_craved_FFQ',
       'cancer_promoting_minus_preventing_liked_FCI',
       'cancer_promoting_minus_preventing_liked_FFQ', 'cancer_promoting_FCI',
       'cancer_promoting_FFQ', 'cancer_preventing_FCI',
       ...
       'FFQ_v2_Mean_Weighted_percent_daily_value',
       'FFQ_v2_Mean_Dietary_Nutrient_Density', 'FFQ_v2_Mean_Energy_Density',
       'FFQ_v2_Mean_Energy', 'wave', 'subj_wave', 'chunks', 'target_y',
       'pred_y', 'mae'],
      dtype='object', length=144)

bf_1

r=0.08; p-value=0.5002

rho=0.04; p-value=0.7340

cancer_promoting_minus_preventing_FFQ

r=0.30; p-value=0.0075

rho=0.32; p-value=0.0039

TESQ_E_sum

r=0.04; p-value=0.7492

rho=-0.02; p-value=0.8590

executive function_association-test_z_FDR_0.01
77.50632911392405

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/scipy/ndimage/measurements.py:305: DeprecationWarning: In future, it will be an error for 'np.bool_' scalars to be interpreted as an index
  return _nd_image.find_objects(input, max_label)

...............................................................................
Index(['subject', 'cancer_promoting_minus_preventing_FCI',
       'cancer_promoting_minus_preventing_FFQ',
       'cancer_promoting_minus_preventing_craved_FCI',
       'cancer_promoting_minus_preventing_craved_FFQ',
       'cancer_promoting_minus_preventing_liked_FCI',
       'cancer_promoting_minus_preventing_liked_FFQ', 'cancer_promoting_FCI',
       'cancer_promoting_FFQ', 'cancer_preventing_FCI',
       ...
       'FFQ_v2_Mean_Weighted_percent_daily_value',
       'FFQ_v2_Mean_Dietary_Nutrient_Density', 'FFQ_v2_Mean_Energy_Density',
       'FFQ_v2_Mean_Energy', 'wave', 'subj_wave', 'chunks', 'target_y',
       'pred_y', 'mae'],
      dtype='object', length=144)

bf_1

r=0.05; p-value=0.7022

rho=-0.03; p-value=0.8363

cancer_promoting_minus_preventing_FFQ

r=0.08; p-value=0.4746

rho=0.11; p-value=0.3731

TESQ_E_sum

r=-0.01; p-value=0.9440

rho=0.01; p-value=0.9522

response inhibition_association-test_z_FDR_0.01
1291.1645569620252

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/scipy/ndimage/measurements.py:305: DeprecationWarning: In future, it will be an error for 'np.bool_' scalars to be interpreted as an index
  return _nd_image.find_objects(input, max_label)

...............................................................................
Index(['subject', 'cancer_promoting_minus_preventing_FCI',
       'cancer_promoting_minus_preventing_FFQ',
       'cancer_promoting_minus_preventing_craved_FCI',
       'cancer_promoting_minus_preventing_craved_FFQ',
       'cancer_promoting_minus_preventing_liked_FCI',
       'cancer_promoting_minus_preventing_liked_FFQ', 'cancer_promoting_FCI',
       'cancer_promoting_FFQ', 'cancer_preventing_FCI',
       ...
       'FFQ_v2_Mean_Weighted_percent_daily_value',
       'FFQ_v2_Mean_Dietary_Nutrient_Density', 'FFQ_v2_Mean_Energy_Density',
       'FFQ_v2_Mean_Energy', 'wave', 'subj_wave', 'chunks', 'target_y',
       'pred_y', 'mae'],
      dtype='object', length=144)

bf_1

r=0.13; p-value=0.2991

rho=0.06; p-value=0.6568

cancer_promoting_minus_preventing_FFQ

r=0.17; p-value=0.1467

rho=0.18; p-value=0.1179

TESQ_E_sum

r=-0.11; p-value=0.3876

rho=-0.08; p-value=0.5041

inhibition_association-test_z_FDR_0.01
436.0886075949367

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/scipy/ndimage/measurements.py:305: DeprecationWarning: In future, it will be an error for 'np.bool_' scalars to be interpreted as an index
  return _nd_image.find_objects(input, max_label)

...............................................................................
Index(['subject', 'cancer_promoting_minus_preventing_FCI',
       'cancer_promoting_minus_preventing_FFQ',
       'cancer_promoting_minus_preventing_craved_FCI',
       'cancer_promoting_minus_preventing_craved_FFQ',
       'cancer_promoting_minus_preventing_liked_FCI',
       'cancer_promoting_minus_preventing_liked_FFQ', 'cancer_promoting_FCI',
       'cancer_promoting_FFQ', 'cancer_preventing_FCI',
       ...
       'FFQ_v2_Mean_Weighted_percent_daily_value',
       'FFQ_v2_Mean_Dietary_Nutrient_Density', 'FFQ_v2_Mean_Energy_Density',
       'FFQ_v2_Mean_Energy', 'wave', 'subj_wave', 'chunks', 'target_y',
       'pred_y', 'mae'],
      dtype='object', length=144)

bf_1

r=-0.00; p-value=0.9830

rho=-0.08; p-value=0.5060

cancer_promoting_minus_preventing_FFQ

r=0.04; p-value=0.7474

rho=0.09; p-value=0.4479

TESQ_E_sum

r=-0.17; p-value=0.1448

rho=-0.21; p-value=0.0728

harvardoxford-cortical_prob_Frontal Orbital Cortex
2847.012658227848

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/scipy/ndimage/measurements.py:305: DeprecationWarning: In future, it will be an error for 'np.bool_' scalars to be interpreted as an index
  return _nd_image.find_objects(input, max_label)

...............................................................................
Index(['subject', 'cancer_promoting_minus_preventing_FCI',
       'cancer_promoting_minus_preventing_FFQ',
       'cancer_promoting_minus_preventing_craved_FCI',
       'cancer_promoting_minus_preventing_craved_FFQ',
       'cancer_promoting_minus_preventing_liked_FCI',
       'cancer_promoting_minus_preventing_liked_FFQ', 'cancer_promoting_FCI',
       'cancer_promoting_FFQ', 'cancer_preventing_FCI',
       ...
       'FFQ_v2_Mean_Weighted_percent_daily_value',
       'FFQ_v2_Mean_Dietary_Nutrient_Density', 'FFQ_v2_Mean_Energy_Density',
       'FFQ_v2_Mean_Energy', 'wave', 'subj_wave', 'chunks', 'target_y',
       'pred_y', 'mae'],
      dtype='object', length=144)

bf_1

r=0.05; p-value=0.6852

rho=0.03; p-value=0.8389

cancer_promoting_minus_preventing_FFQ

r=0.14; p-value=0.2242

rho=0.19; p-value=0.0947

TESQ_E_sum

r=-0.00; p-value=0.9902

rho=-0.06; p-value=0.6249

harvardoxford-cortical_prob_Inferior Frontal Gyrus, pars opercularis
1471.4303797468353

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/scipy/ndimage/measurements.py:305: DeprecationWarning: In future, it will be an error for 'np.bool_' scalars to be interpreted as an index
  return _nd_image.find_objects(input, max_label)

...............................................................................
Index(['subject', 'cancer_promoting_minus_preventing_FCI',
       'cancer_promoting_minus_preventing_FFQ',
       'cancer_promoting_minus_preventing_craved_FCI',
       'cancer_promoting_minus_preventing_craved_FFQ',
       'cancer_promoting_minus_preventing_liked_FCI',
       'cancer_promoting_minus_preventing_liked_FFQ', 'cancer_promoting_FCI',
       'cancer_promoting_FFQ', 'cancer_preventing_FCI',
       ...
       'FFQ_v2_Mean_Weighted_percent_daily_value',
       'FFQ_v2_Mean_Dietary_Nutrient_Density', 'FFQ_v2_Mean_Energy_Density',
       'FFQ_v2_Mean_Energy', 'wave', 'subj_wave', 'chunks', 'target_y',
       'pred_y', 'mae'],
      dtype='object', length=144)

bf_1

r=0.09; p-value=0.4700

rho=0.09; p-value=0.4461

cancer_promoting_minus_preventing_FFQ

r=0.15; p-value=0.1912

rho=0.09; p-value=0.4369

TESQ_E_sum

r=-0.33; p-value=0.0057

rho=-0.30; p-value=0.0124

harvardoxford-cortical_prob_Inferior Frontal Gyrus, pars triangularis
1538.620253164557

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/scipy/ndimage/measurements.py:305: DeprecationWarning: In future, it will be an error for 'np.bool_' scalars to be interpreted as an index
  return _nd_image.find_objects(input, max_label)

...............................................................................
Index(['subject', 'cancer_promoting_minus_preventing_FCI',
       'cancer_promoting_minus_preventing_FFQ',
       'cancer_promoting_minus_preventing_craved_FCI',
       'cancer_promoting_minus_preventing_craved_FFQ',
       'cancer_promoting_minus_preventing_liked_FCI',
       'cancer_promoting_minus_preventing_liked_FFQ', 'cancer_promoting_FCI',
       'cancer_promoting_FFQ', 'cancer_preventing_FCI',
       ...
       'FFQ_v2_Mean_Weighted_percent_daily_value',
       'FFQ_v2_Mean_Dietary_Nutrient_Density', 'FFQ_v2_Mean_Energy_Density',
       'FFQ_v2_Mean_Energy', 'wave', 'subj_wave', 'chunks', 'target_y',
       'pred_y', 'mae'],
      dtype='object', length=144)

bf_1

r=0.01; p-value=0.9557

rho=-0.04; p-value=0.7644

cancer_promoting_minus_preventing_FFQ

r=0.04; p-value=0.7313

rho=0.05; p-value=0.6446

TESQ_E_sum

r=-0.10; p-value=0.3993

rho=-0.15; p-value=0.2086

brain_data_filepath = (
    "/gpfs/projects/sanlab/shared/DEV/nonbids_data/fMRI/ml/SST/mvpa_Dataset_conditions_84subs_correct_stop_go_contrast.pkl"
)

target='SST_SSRT'

with open(brain_data_filepath, 'rb') as pkl_file:
    Brain_Data_allsubs = pickle.load(pkl_file)

Brain_Data_allsubs = Brain_Data_allsubs[np.isnan(Brain_Data_allsubs.sa[target].value)==False]

print(mask_name)

Brain_Data_allsubs = setup_metadata(Brain_Data_allsubs,target,standardize=True)

print(sum(sum(Brain_Data_allsubs.samples>0))/len(Brain_Data_allsubs.samples))

masked_preview = unmask(Brain_Data_allsubs.samples[0], Brain_Data_allsubs.a['mask_nifti'].value)
plotting.plot_stat_map(masked_preview,vmax=18)
plotting.show()
forced_choice_results = do_regression(Brain_Data_allsubs,get_predict_and_prob=do_Ridge)
print("")


forced_choice_results_dict[mask_name]=forced_choice_results
print_regression_results(forced_choice_results,['mae'])

harvardoxford-cortical_prob_Inferior Frontal Gyrus, pars triangularis
108520.02531645569

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/scipy/ndimage/measurements.py:305: DeprecationWarning: In future, it will be an error for 'np.bool_' scalars to be interpreted as an index
  return _nd_image.find_objects(input, max_label)

..

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.3466e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)

..

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.55211e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)
/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.27406e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)

..

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.11435e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)
/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.01995e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)

..

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.22174e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)
/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.1365e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)

..

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.29197e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)
/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.37195e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)

..

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.14587e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)
/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.3401e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)

..

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.4792e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)
/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.13074e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)

..

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.51638e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)
/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.10771e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)

..

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.42884e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)
/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.16605e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)

..

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.24135e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)
/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.27849e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)

..

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.17444e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)
/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.51536e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)

..

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.03639e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)
/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.32662e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)

..

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.47439e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)
/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.263e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)

..

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.14526e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)
/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.24325e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)

..

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.36e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)
/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.33944e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)

..

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.41993e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)
/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.16458e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)

..

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.16236e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)
/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.27412e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)

..

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.81581e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)
/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.41303e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)

..

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.29131e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)
/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.22977e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)

..

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=2.99843e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)
/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.44902e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)

..

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=2.88395e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)
/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.28041e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)

..

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.1843e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)
/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.78835e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)

..

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.16703e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)
/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.57511e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)

..

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.42558e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)
/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.4367e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)

..

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.63866e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)
/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.40746e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)

..

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.23091e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)
/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.29997e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)

..

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.22218e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)
/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.05825e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)

..

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.30593e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)
/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.46081e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)

..

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.24517e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)
/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.43528e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)

..

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.42074e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)
/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=2.99017e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)

..

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.1692e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)
/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.41775e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)

..

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.22715e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)
/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.34319e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)

..

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.15753e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)
/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.10879e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)

..

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.24447e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)
/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.27121e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)

..

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.22157e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)
/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.27406e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)

..

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.06374e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)
/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.056e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)

..

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.48659e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)
/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.05474e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)

..

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.16402e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)
/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.22216e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)

..

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.14187e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)
/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.29135e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)

.

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.13431e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)
/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.32289e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)

Index(['subject', 'cancer_promoting_minus_preventing_FCI',
       'cancer_promoting_minus_preventing_FFQ',
       'cancer_promoting_minus_preventing_craved_FCI',
       'cancer_promoting_minus_preventing_craved_FFQ',
       'cancer_promoting_minus_preventing_liked_FCI',
       'cancer_promoting_minus_preventing_liked_FFQ', 'cancer_promoting_FCI',
       'cancer_promoting_FFQ', 'cancer_preventing_FCI',
       ...
       'FFQ_v2_Mean_Weighted_percent_daily_value',
       'FFQ_v2_Mean_Dietary_Nutrient_Density', 'FFQ_v2_Mean_Energy_Density',
       'FFQ_v2_Mean_Energy', 'wave', 'subj_wave', 'chunks', 'target_y',
       'pred_y', 'mae'],
      dtype='object', length=144)

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/sklearn/linear_model/_ridge.py:194: LinAlgWarning: Ill-conditioned matrix (rcond=3.00275e-08): result may not be accurate.
  dual_coef = linalg.solve(K, y, sym_pos=True, overwrite_a=False)

bf_1

r=0.11; p-value=0.3682

rho=0.06; p-value=0.6085

cancer_promoting_minus_preventing_FFQ

r=0.24; p-value=0.0355

rho=0.25; p-value=0.0234

TESQ_E_sum

r=-0.14; p-value=0.2277

rho=-0.20; p-value=0.0828

mae_predictivity_wholebrain = forced_choice_results

Test 2

individual_differences = pd.read_csv(ml_data_folderpath + "/data_by_ppt_2022_02_26.csv")
individual_differences = individual_differences.rename(columns={'SID':'subject'})
individual_differences['wave']=1

brain_data_filepath = (
    "/gpfs/projects/sanlab/shared/DEV/nonbids_data/fMRI/ml/SST/mvpa_Dataset_conditions_84subs_correct_cond.pkl"
)

with open(brain_data_filepath, 'rb') as pkl_file:
    Brain_Data_allsubs = pickle.load(pkl_file)

Brain_Data_allsubs = setup_metadata(Brain_Data_allsubs)

So actually, probably neural activity explains additional variance in a very straightforward way.

We found some relationships with FFQ. If these hold, it would be interesting to see support from them.

sooo...let's get a record of all the items from test 1 that we want to try running...

def select_CS_trials(df):
    return(df[df['target_y']=='CorrectStop'])

def select_CS_and_labels(df):
    return(df.loc[:,['chunks','pred_prob_CorrectStop']])

def label_cols(df, prepend):
    df1=df.copy()
    df1.columns = [prepend + c if c!="chunks" else c for c in df.columns]
    return(df1)

linearSVC_cor_wholebrain_CS_CS = label_cols(prepend="linearSVC_cor_wholebrain_CS_CS", df=select_CS_and_labels(select_CS_trials(LinearSVC_correlation_wholebrain['sample_wise'])))
linearSVC_cor_planning_CS_CS = label_cols(prepend="linearSVC_cor_planning_CS_CS", df=select_CS_and_labels(select_CS_trials(LinearSVC_correlation_masked['planning_association-test_z_FDR_0.01']['sample_wise'])))
SVC_prob_planning_CS_CS = label_cols(prepend="SVC_prob_planning_CS_CS", df=select_CS_and_labels(select_CS_trials(SVC_proba_masked['planning_association-test_z_FDR_0.01']['sample_wise'])))
SVC_prob_exec_function_CS_CS = label_cols(prepend="SVC_prob_exec_function_CS_CS", df=select_CS_and_labels(select_CS_trials(SVC_proba_masked['executive function_association-test_z_FDR_0.01']['sample_wise'])))
SVC_inhib_CS_CS = label_cols(prepend="SVC_inhib_CS_CS", df=select_CS_and_labels(select_CS_trials(SVC_proba_masked['inhibition_association-test_z_FDR_0.01']['sample_wise'])))
SVC_IFG_PT_CS_CS = label_cols(prepend="SVC_IFG_PT_CS_CS", df=select_CS_and_labels(select_CS_trials(SVC_proba_masked['harvardoxford-cortical_prob_Inferior Frontal Gyrus, pars triangularis']['sample_wise'])))
SVC_wholebrain_CS_CS = label_cols(prepend="SVC_wholebrain_CS_CS", df=select_CS_and_labels(select_CS_trials(SVC_proba_wholebrain['sample_wise'])))

neural_all = pd.merge(pd.merge(pd.merge(pd.merge(pd.merge(pd.merge(
        linearSVC_cor_wholebrain_CS_CS,
        linearSVC_cor_planning_CS_CS),
        SVC_prob_planning_CS_CS),
        SVC_prob_exec_function_CS_CS),
        SVC_inhib_CS_CS),
        SVC_IFG_PT_CS_CS),
        SVC_wholebrain_CS_CS)

individual_differences['subj_wave'] = individual_differences.subject + "_" + individual_differences.wave.astype(str)

neural_behavioral = pd.merge(
    neural_all,
    individual_differences,
    left_on='chunks',right_on='subj_wave')

neural_behavioral.to_csv(ml_data_folderpath + "/analysis_v5_test1.csv")

ml_data_folderpath

'/gpfs/projects/sanlab/shared/DEV/nonbids_data/fMRI/ml'

PES-related

Discriminability

pes_vars_of_interest  =['RTFS_factor_1','RTFS_factor_2','RTFS_f1_minus_f2', 
                        'SST_PostErrorSlowW1_median','SST_PostErrorSlowW1_mean','SST_pes_mean_limited',
                        'FFQ_v2_Mean_Weighted_Nutrient_Density']

all_masks = get_failure_masks(ml_data_folderpath)

PES-related conditions separate

whole-brain:

np.random.seed(1646100326)

print("whole brain")
brain_data_filepath = (
    "/gpfs/projects/sanlab/shared/DEV/nonbids_data/fMRI/ml/SST/mvpa_Dataset_posterror_conditions_84subs_post_stop_cg.pkl"
)

with open(brain_data_filepath, 'rb') as pkl_file:
    Brain_Data_allsubs = pickle.load(pkl_file)

Brain_Data_allsubs = setup_metadata(Brain_Data_allsubs)

print(sum(sum(Brain_Data_allsubs.samples>0))/len(Brain_Data_allsubs.samples))

#masked_preview = unmask(Brain_Data_allsubs.samples[0], Brain_Data_allsubs.a['mask_nifti'].value)
#plotting.plot_stat_map(masked_preview,vmax=18)

forced_choice_results = do_forced_choice(Brain_Data_allsubs,get_predict_and_prob=do_SVC)

whole brain
124651.33333333333
.................................................................................

forced_choice_results['sample_wise']['ln_inv_pred_prob_CGAfterCS']=inv_log_transform(forced_choice_results['sample_wise']['pred_prob_CorrectGoFollowingCorrectStop'])
forced_choice_results['sample_wise']['ln_inv_pred_prob_CGAfterFS']=inv_log_transform(forced_choice_results['sample_wise']['pred_prob_CorrectGoFollowingFailedStop'])


sample_wise_results = forced_choice_results['sample_wise']
prediction = np.mean(forced_choice_results['sample_wise']['target_y']==forced_choice_results['sample_wise']['pred_y'])
forced_choice_prediction = np.mean(forced_choice_results['sample_wise']['target_y']==forced_choice_results['sample_wise']['pred_y_forced_choice'])


print_forced_choice_results(
    forced_choice_results,
    individual_differences,
    ['ln_inv_pred_prob_CGAfterCS'],
    behavioral_measures = pes_vars_of_interest,
    target_y='CorrectGoFollowingCorrectStop')
print_forced_choice_results(
    forced_choice_results,
    individual_differences,
    ['ln_inv_pred_prob_CGAfterFS'],
    behavioral_measures = pes_vars_of_interest,
    target_y='CorrectGoFollowingFailedStop')

print(prediction,forced_choice_prediction)

(0.5308641975308642, 0.35802469135802467)

RTFS_factor_1

r=0.28; p-value=0.0608

rho=0.30; p-value=0.0428

RTFS_factor_2

r=-0.18; p-value=0.2387

rho=-0.13; p-value=0.3746

RTFS_f1_minus_f2

r=0.35; p-value=0.0215

rho=0.31; p-value=0.0440

SST_PostErrorSlowW1_median

r=-0.07; p-value=0.5857

rho=-0.07; p-value=0.6127

SST_PostErrorSlowW1_mean

r=-0.12; p-value=0.3576

rho=-0.09; p-value=0.4876

SST_pes_mean_limited

r=-0.13; p-value=0.3068

rho=-0.15; p-value=0.2487

FFQ_v2_Mean_Weighted_Nutrient_Density

r=0.01; p-value=0.9584

rho=-0.00; p-value=0.9988

(0.5308641975308642, 0.35802469135802467)

RTFS_factor_1

r=-0.26; p-value=0.0846

rho=-0.18; p-value=0.2403

RTFS_factor_2

r=0.03; p-value=0.8676

rho=-0.06; p-value=0.6794

RTFS_f1_minus_f2

r=-0.20; p-value=0.1865

rho=-0.09; p-value=0.5696

SST_PostErrorSlowW1_median

r=0.16; p-value=0.2330

rho=0.09; p-value=0.4848

SST_PostErrorSlowW1_mean

r=0.08; p-value=0.5466

rho=0.02; p-value=0.9033

SST_pes_mean_limited

r=-0.04; p-value=0.7371

rho=-0.03; p-value=0.8308

FFQ_v2_Mean_Weighted_Nutrient_Density

r=-0.12; p-value=0.3463

rho=-0.16; p-value=0.2015

0.5308641975308642 0.35802469135802467

print("whole brain")
brain_data_filepath = (
    "/gpfs/projects/sanlab/shared/DEV/nonbids_data/fMRI/ml/SST/mvpa_Dataset_posterror_conditions_84subs_post_stop_cg.pkl"
)

with open(brain_data_filepath, 'rb') as pkl_file:
    Brain_Data_allsubs = pickle.load(pkl_file)

Brain_Data_allsubs = setup_metadata(Brain_Data_allsubs)

print(sum(sum(Brain_Data_allsubs.samples>0))/len(Brain_Data_allsubs.samples))

#masked_preview = unmask(Brain_Data_allsubs.samples[0], Brain_Data_allsubs.a['mask_nifti'].value)
#plotting.plot_stat_map(masked_preview,vmax=18)

forced_choice_results = do_forced_choice(Brain_Data_allsubs,get_predict_and_prob=do_SVC)


forced_choice_results['sample_wise']['ln_inv_pred_prob_CGAfterCS']=inv_log_transform(forced_choice_results['sample_wise']['pred_prob_CorrectGoFollowingCorrectStop'])
forced_choice_results['sample_wise']['ln_inv_pred_prob_CGAfterFS']=inv_log_transform(forced_choice_results['sample_wise']['pred_prob_CorrectGoFollowingFailedStop'])


sample_wise_results = forced_choice_results['sample_wise']
prediction = np.mean(forced_choice_results['sample_wise']['target_y']==forced_choice_results['sample_wise']['pred_y'])
forced_choice_prediction = np.mean(forced_choice_results['sample_wise']['target_y']==forced_choice_results['sample_wise']['pred_y_forced_choice'])

print(prediction,forced_choice_prediction)

print_forced_choice_results(
    forced_choice_results,
    individual_differences,
    ['ln_inv_pred_prob_CGAfterCS'],
    behavioral_measures = pes_vars_of_interest,
    target_y='CorrectGoFollowingCorrectStop')
print_forced_choice_results(
    forced_choice_results,
    individual_differences,
    ['ln_inv_pred_prob_CGAfterFS'],
    behavioral_measures = pes_vars_of_interest,
    target_y='CorrectGoFollowingFailedStop')

print(prediction,forced_choice_prediction)

whole brain
124651.33333333333
.................................................................................0.5308641975308642 0.35802469135802467
(0.5308641975308642, 0.35802469135802467)

RTFS_factor_1

r=0.19; p-value=0.1992

rho=0.17; p-value=0.2547

RTFS_factor_2

r=0.01; p-value=0.9216

rho=0.01; p-value=0.9409

RTFS_f1_minus_f2

r=0.12; p-value=0.4171

rho=0.08; p-value=0.5938

SST_PostErrorSlowW1_median

r=0.14; p-value=0.2850

rho=0.08; p-value=0.5617

SST_PostErrorSlowW1_mean

r=0.08; p-value=0.5577

rho=0.04; p-value=0.7320

SST_pes_mean_limited

r=-0.09; p-value=0.4742

rho=-0.15; p-value=0.2593

FFQ_v2_Mean_Weighted_Nutrient_Density

r=0.06; p-value=0.6375

rho=0.01; p-value=0.9623

(0.5308641975308642, 0.35802469135802467)

RTFS_factor_1

r=0.15; p-value=0.2908

rho=0.13; p-value=0.3854

RTFS_factor_2

r=-0.04; p-value=0.7793

rho=-0.03; p-value=0.8503

RTFS_f1_minus_f2

r=0.15; p-value=0.3297

rho=0.10; p-value=0.5068

SST_PostErrorSlowW1_median

r=-0.04; p-value=0.7588

rho=0.03; p-value=0.8018

SST_PostErrorSlowW1_mean

r=-0.10; p-value=0.4493

rho=-0.09; p-value=0.4818

SST_pes_mean_limited

r=-0.15; p-value=0.2546

rho=-0.11; p-value=0.3750

FFQ_v2_Mean_Weighted_Nutrient_Density

r=0.05; p-value=0.6760

rho=0.05; p-value=0.6816

0.5308641975308642 0.35802469135802467

forced_choice_results['sample_wise']

	chunks	target_y	pred_y	pred_y_forced_choice	pred_prob_CorrectGoFollowingCorrectStop	pred_prob_CorrectGoFollowingFailedStop	ln_inv_pred_prob_CGAfterCS	ln_inv_pred_prob_CGAfterFS
0	DEV005_1	CorrectGoFollowingCorrectStop	CorrectGoFollowingCorrectStop	CorrectGoFollowingFailedStop	0.491896	0.508104	-0.677069	-0.709488
1	DEV005_1	CorrectGoFollowingFailedStop	CorrectGoFollowingCorrectStop	CorrectGoFollowingCorrectStop	0.494236	0.505764	-0.681684	-0.704743
0	DEV006_1	CorrectGoFollowingCorrectStop	CorrectGoFollowingCorrectStop	CorrectGoFollowingFailedStop	0.493639	0.506361	-0.680504	-0.705952
1	DEV006_1	CorrectGoFollowingFailedStop	CorrectGoFollowingFailedStop	CorrectGoFollowingCorrectStop	0.507756	0.492244	-0.708780	-0.677755
0	DEV010_1	CorrectGoFollowingCorrectStop	CorrectGoFollowingCorrectStop	CorrectGoFollowingCorrectStop	0.461919	0.538081	-0.619745	-0.772367
...	...	...	...	...	...	...	...	...
1	DEV216_1	CorrectGoFollowingFailedStop	CorrectGoFollowingFailedStop	CorrectGoFollowingCorrectStop	0.492047	0.507953	-0.677365	-0.709182
0	DEV217_1	CorrectGoFollowingCorrectStop	CorrectGoFollowingCorrectStop	CorrectGoFollowingFailedStop	0.432470	0.567530	-0.566461	-0.838243
1	DEV217_1	CorrectGoFollowingFailedStop	CorrectGoFollowingCorrectStop	CorrectGoFollowingCorrectStop	0.444805	0.555195	-0.588436	-0.810119
0	DEV218_1	CorrectGoFollowingCorrectStop	CorrectGoFollowingCorrectStop	CorrectGoFollowingCorrectStop	0.500000	0.500000	-0.693147	-0.693147
1	DEV218_1	CorrectGoFollowingFailedStop	CorrectGoFollowingCorrectStop	CorrectGoFollowingFailedStop	0.500000	0.500000	-0.693147	-0.693147

162 rows × 8 columns

SVC_proba_wholebrain = forced_choice_results

masked:

forced_choice_results_dict = {}
for m_i,m_r in all_masks.iterrows():
    mask_name=m_r['mask_name']
    brain_data_filepath = (
        "/gpfs/projects/sanlab/shared/DEV/nonbids_data/fMRI/ml/SST/mvpa_Dataset_posterror_conditions_84subs_post_stop_cg_" +
        mask_name
        + ".pkl"
    )
    print(brain_data_filepath)
    if os.path.exists(brain_data_filepath):
        with open(brain_data_filepath, 'rb') as pkl_file:
            Brain_Data_allsubs = pickle.load(pkl_file)
            
        print(mask_name)

        Brain_Data_allsubs = setup_metadata(Brain_Data_allsubs)
        
        print(sum(sum(Brain_Data_allsubs.samples>0))/len(Brain_Data_allsubs.samples))
        
        masked_preview = unmask(Brain_Data_allsubs.samples[0], Brain_Data_allsubs.a['mask_nifti'].value)
        plotting.plot_stat_map(masked_preview,vmax=18)
        plotting.show()
        forced_choice_results = do_forced_choice(Brain_Data_allsubs,get_predict_and_prob=do_SVC)
        
        
        prediction = np.mean(forced_choice_results['sample_wise']['target_y']==forced_choice_results['sample_wise']['pred_y'])
        forced_choice_prediction = np.mean(forced_choice_results['sample_wise']['target_y']==forced_choice_results['sample_wise']['pred_y_forced_choice'])
        
        forced_choice_results['sample_wise']['ln_inv_pred_prob_CGAfterCS']=inv_log_transform(forced_choice_results['sample_wise']['pred_prob_CorrectGoFollowingCorrectStop'])
        forced_choice_results['sample_wise']['ln_inv_pred_prob_CGAfterFS']=inv_log_transform(forced_choice_results['sample_wise']['pred_prob_CorrectGoFollowingFailedStop'])

        print(prediction,forced_choice_prediction)

        forced_choice_results_dict[mask_name]=forced_choice_results

        
        print_forced_choice_results(
            forced_choice_results,
            individual_differences,
            ['ln_inv_pred_prob_CGAfterCS'],
            behavioral_measures = pes_vars_of_interest,
            target_y='CorrectGoFollowingCorrectStop')
        print_forced_choice_results(
            forced_choice_results,
            individual_differences,
            ['ln_inv_pred_prob_CGAfterFS'],
            behavioral_measures = pes_vars_of_interest,
            target_y='CorrectGoFollowingFailedStop')

/gpfs/projects/sanlab/shared/DEV/nonbids_data/fMRI/ml/SST/mvpa_Dataset_posterror_conditions_84subs_post_stop_cg_negative affect_association-test_z_FDR_0.01.pkl
negative affect_association-test_z_FDR_0.01
1209.0617283950617

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/nilearn/plotting/img_plotting.py:348: FutureWarning: Default resolution of the MNI template will change from 2mm to 1mm in version 0.10.0
  anat_img = load_mni152_template()
/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/scipy/ndimage/measurements.py:305: DeprecationWarning: In future, it will be an error for 'np.bool_' scalars to be interpreted as an index
  return _nd_image.find_objects(input, max_label)

.................................................................................0.5 0.30864197530864196
(0.5, 0.30864197530864196)

RTFS_factor_1

r=0.14; p-value=0.3267

rho=0.18; p-value=0.1991

RTFS_factor_2

r=0.17; p-value=0.2400

rho=0.01; p-value=0.9628

RTFS_f1_minus_f2

r=-0.03; p-value=0.8505

rho=0.08; p-value=0.5790

SST_PostErrorSlowW1_median

r=-0.09; p-value=0.4719

rho=-0.12; p-value=0.3364

SST_PostErrorSlowW1_mean

r=-0.03; p-value=0.8060

rho=-0.04; p-value=0.7264

SST_pes_mean_limited

r=-0.07; p-value=0.5807

rho=-0.00; p-value=0.9788

FFQ_v2_Mean_Weighted_Nutrient_Density

r=0.02; p-value=0.8576

rho=-0.02; p-value=0.8449

(0.5, 0.30864197530864196)

RTFS_factor_1

r=-0.07; p-value=0.6195

rho=-0.11; p-value=0.4347

RTFS_factor_2

r=-0.02; p-value=0.8626

rho=-0.04; p-value=0.7931

RTFS_f1_minus_f2

r=-0.05; p-value=0.7506

rho=-0.07; p-value=0.6347

SST_PostErrorSlowW1_median

r=0.05; p-value=0.7216

rho=0.06; p-value=0.6243

SST_PostErrorSlowW1_mean

r=-0.08; p-value=0.5485

rho=-0.07; p-value=0.6095

SST_pes_mean_limited

r=-0.10; p-value=0.4404

rho=-0.11; p-value=0.3857

FFQ_v2_Mean_Weighted_Nutrient_Density

r=-0.03; p-value=0.7966

rho=0.01; p-value=0.9144

/gpfs/projects/sanlab/shared/DEV/nonbids_data/fMRI/ml/SST/mvpa_Dataset_posterror_conditions_84subs_post_stop_cg_emotion regulation_association-test_z_FDR_0.01.pkl
emotion regulation_association-test_z_FDR_0.01
956.5493827160494

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/scipy/ndimage/measurements.py:305: DeprecationWarning: In future, it will be an error for 'np.bool_' scalars to be interpreted as an index
  return _nd_image.find_objects(input, max_label)

.................................................................................0.5061728395061729 0.32098765432098764
(0.5061728395061729, 0.32098765432098764)

RTFS_factor_1

r=0.03; p-value=0.8331

rho=0.10; p-value=0.5011

RTFS_factor_2

r=0.34; p-value=0.0160

rho=0.26; p-value=0.0691

RTFS_f1_minus_f2

r=-0.25; p-value=0.0866

rho=-0.17; p-value=0.2494

SST_PostErrorSlowW1_median

r=0.04; p-value=0.7864

rho=-0.02; p-value=0.8874

SST_PostErrorSlowW1_mean

r=0.09; p-value=0.4648

rho=0.06; p-value=0.6312

SST_pes_mean_limited

r=-0.02; p-value=0.8943

rho=-0.03; p-value=0.7908

FFQ_v2_Mean_Weighted_Nutrient_Density

r=-0.00; p-value=0.9790

rho=0.01; p-value=0.9362

(0.5061728395061729, 0.32098765432098764)

RTFS_factor_1

r=0.11; p-value=0.4574

rho=0.07; p-value=0.6159

RTFS_factor_2

r=-0.10; p-value=0.5021

rho=-0.13; p-value=0.3748

RTFS_f1_minus_f2

r=0.16; p-value=0.2630

rho=0.12; p-value=0.3992

SST_PostErrorSlowW1_median

r=-0.04; p-value=0.7921

rho=-0.03; p-value=0.8320

SST_PostErrorSlowW1_mean

r=-0.24; p-value=0.0608

rho=-0.22; p-value=0.0863

SST_pes_mean_limited

r=-0.12; p-value=0.3605

rho=-0.05; p-value=0.6893

FFQ_v2_Mean_Weighted_Nutrient_Density

r=0.02; p-value=0.8484

rho=-0.02; p-value=0.8926

/gpfs/projects/sanlab/shared/DEV/nonbids_data/fMRI/ml/SST/mvpa_Dataset_posterror_conditions_84subs_post_stop_cg_harvardoxford-cortical_prob_Frontal Medial Cortex.pkl
harvardoxford-cortical_prob_Frontal Medial Cortex
1744.5

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/scipy/ndimage/measurements.py:305: DeprecationWarning: In future, it will be an error for 'np.bool_' scalars to be interpreted as an index
  return _nd_image.find_objects(input, max_label)

.................................................................................0.5 0.4444444444444444
(0.5, 0.4444444444444444)

RTFS_factor_1

r=-0.11; p-value=0.4554

rho=-0.13; p-value=0.3727

RTFS_factor_2

r=0.16; p-value=0.2581

rho=0.16; p-value=0.2570

RTFS_f1_minus_f2

r=-0.25; p-value=0.0863

rho=-0.21; p-value=0.1585

SST_PostErrorSlowW1_median

r=0.02; p-value=0.8529

rho=0.04; p-value=0.7597

SST_PostErrorSlowW1_mean

r=0.10; p-value=0.4274

rho=0.13; p-value=0.3166

SST_pes_mean_limited

r=0.08; p-value=0.5456

rho=0.12; p-value=0.3594

FFQ_v2_Mean_Weighted_Nutrient_Density

r=-0.14; p-value=0.2341

rho=-0.11; p-value=0.3442

(0.5, 0.4444444444444444)

RTFS_factor_1

r=0.11; p-value=0.4396

rho=0.11; p-value=0.4450

RTFS_factor_2

r=-0.21; p-value=0.1498

rho=-0.25; p-value=0.0810

RTFS_f1_minus_f2

r=0.29; p-value=0.0464

rho=0.30; p-value=0.0396

SST_PostErrorSlowW1_median

r=-0.08; p-value=0.5213

rho=-0.08; p-value=0.5646

SST_PostErrorSlowW1_mean

r=-0.13; p-value=0.3026

rho=-0.13; p-value=0.3166

SST_pes_mean_limited

r=-0.08; p-value=0.5567

rho=-0.07; p-value=0.5752

FFQ_v2_Mean_Weighted_Nutrient_Density

r=0.18; p-value=0.1437

rho=0.19; p-value=0.1203

/gpfs/projects/sanlab/shared/DEV/nonbids_data/fMRI/ml/SST/mvpa_Dataset_posterror_conditions_84subs_post_stop_cg_negative emotions_association-test_z_FDR_0.01.pkl
negative emotions_association-test_z_FDR_0.01
575.395061728395

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/scipy/ndimage/measurements.py:305: DeprecationWarning: In future, it will be an error for 'np.bool_' scalars to be interpreted as an index
  return _nd_image.find_objects(input, max_label)

.................................................................................0.5 0.37037037037037035
(0.5, 0.37037037037037035)

RTFS_factor_1

r=0.07; p-value=0.6294

rho=0.07; p-value=0.6184

RTFS_factor_2

r=-0.17; p-value=0.2462

rho=-0.12; p-value=0.3944

RTFS_f1_minus_f2

r=0.18; p-value=0.2096

rho=0.15; p-value=0.3043

SST_PostErrorSlowW1_median

r=-0.13; p-value=0.3138

rho=-0.16; p-value=0.2047

SST_PostErrorSlowW1_mean

r=-0.01; p-value=0.9554

rho=-0.05; p-value=0.6815

SST_pes_mean_limited

r=-0.11; p-value=0.3946

rho=-0.11; p-value=0.3643

FFQ_v2_Mean_Weighted_Nutrient_Density

r=0.04; p-value=0.7479

rho=-0.03; p-value=0.7730

(0.5, 0.37037037037037035)

RTFS_factor_1

r=-0.11; p-value=0.4720

rho=-0.08; p-value=0.5762

RTFS_factor_2

r=0.19; p-value=0.1989

rho=0.16; p-value=0.2660

RTFS_f1_minus_f2

r=-0.22; p-value=0.1454

rho=-0.20; p-value=0.1705

SST_PostErrorSlowW1_median

r=0.11; p-value=0.3922

rho=0.20; p-value=0.1401

SST_PostErrorSlowW1_mean

r=-0.13; p-value=0.3380

rho=-0.12; p-value=0.3598

SST_pes_mean_limited

r=0.06; p-value=0.6543

rho=0.05; p-value=0.7261

FFQ_v2_Mean_Weighted_Nutrient_Density

r=0.03; p-value=0.8371

rho=0.06; p-value=0.6004

/gpfs/projects/sanlab/shared/DEV/nonbids_data/fMRI/ml/SST/mvpa_Dataset_posterror_conditions_84subs_post_stop_cg_self referential_association-test_z_FDR_0.01.pkl
self referential_association-test_z_FDR_0.01
1752.2592592592594

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/scipy/ndimage/measurements.py:305: DeprecationWarning: In future, it will be an error for 'np.bool_' scalars to be interpreted as an index
  return _nd_image.find_objects(input, max_label)

.................................................................................0.5 0.37037037037037035
(0.5, 0.37037037037037035)

RTFS_factor_1

r=-0.07; p-value=0.6212

rho=-0.04; p-value=0.7904

RTFS_factor_2

r=-0.24; p-value=0.0849

rho=-0.16; p-value=0.2648

RTFS_f1_minus_f2

r=0.16; p-value=0.2635

rho=0.17; p-value=0.2341

SST_PostErrorSlowW1_median

r=-0.02; p-value=0.8498

rho=0.05; p-value=0.7074

SST_PostErrorSlowW1_mean

r=-0.12; p-value=0.3455

rho=-0.13; p-value=0.3123

SST_pes_mean_limited

r=-0.11; p-value=0.3705

rho=-0.08; p-value=0.5090

FFQ_v2_Mean_Weighted_Nutrient_Density

r=0.20; p-value=0.0974

rho=0.10; p-value=0.3892

(0.5, 0.37037037037037035)

RTFS_factor_1

r=-0.09; p-value=0.5458

rho=-0.05; p-value=0.7186

RTFS_factor_2

r=0.07; p-value=0.6377

rho=0.04; p-value=0.7700

RTFS_f1_minus_f2

r=-0.13; p-value=0.4053

rho=-0.11; p-value=0.4710

SST_PostErrorSlowW1_median

r=-0.01; p-value=0.9387

rho=-0.07; p-value=0.6174

SST_PostErrorSlowW1_mean

r=0.10; p-value=0.4171

rho=0.09; p-value=0.4650

SST_pes_mean_limited

r=0.09; p-value=0.4981

rho=0.07; p-value=0.6056

FFQ_v2_Mean_Weighted_Nutrient_Density

r=-0.05; p-value=0.6681

rho=0.02; p-value=0.8478

/gpfs/projects/sanlab/shared/DEV/nonbids_data/fMRI/ml/SST/mvpa_Dataset_posterror_conditions_84subs_post_stop_cg_harvardoxford-cortical_prob_Frontal Orbital Cortex.pkl
harvardoxford-cortical_prob_Frontal Orbital Cortex
2879.141975308642

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/scipy/ndimage/measurements.py:305: DeprecationWarning: In future, it will be an error for 'np.bool_' scalars to be interpreted as an index
  return _nd_image.find_objects(input, max_label)

.................................................................................0.5123456790123457 0.2839506172839506
(0.5123456790123457, 0.2839506172839506)

RTFS_factor_1

r=-0.24; p-value=0.0966

rho=-0.18; p-value=0.2138

RTFS_factor_2

r=0.00; p-value=0.9776

rho=0.05; p-value=0.7574

RTFS_f1_minus_f2

r=-0.12; p-value=0.4230

rho=-0.11; p-value=0.4737

SST_PostErrorSlowW1_median

r=0.23; p-value=0.0794

rho=0.21; p-value=0.1049

SST_PostErrorSlowW1_mean

r=0.19; p-value=0.1393

rho=0.20; p-value=0.1258

SST_pes_mean_limited

r=0.16; p-value=0.1997

rho=0.16; p-value=0.2215

FFQ_v2_Mean_Weighted_Nutrient_Density

r=0.12; p-value=0.3395

rho=0.10; p-value=0.4023

(0.5123456790123457, 0.2839506172839506)

RTFS_factor_1

r=0.36; p-value=0.0111

rho=0.31; p-value=0.0291

RTFS_factor_2

r=0.03; p-value=0.8575

rho=0.06; p-value=0.6721

RTFS_f1_minus_f2

r=0.19; p-value=0.1919

rho=0.10; p-value=0.5087

SST_PostErrorSlowW1_median

r=-0.18; p-value=0.1628

rho=-0.17; p-value=0.1963

SST_PostErrorSlowW1_mean

r=-0.08; p-value=0.5577

rho=-0.07; p-value=0.5719

SST_pes_mean_limited

r=-0.18; p-value=0.1591

rho=-0.15; p-value=0.2496

FFQ_v2_Mean_Weighted_Nutrient_Density

r=-0.01; p-value=0.9087

rho=-0.01; p-value=0.9403

/gpfs/projects/sanlab/shared/DEV/nonbids_data/fMRI/ml/SST/mvpa_Dataset_posterror_conditions_84subs_post_stop_cg_harvardoxford-cortical_prob_Cingulate Gyrus, anterior division.pkl
harvardoxford-cortical_prob_Cingulate Gyrus, anterior division
2791.222222222222

/home/bsmith16/.conda/envs/py3_mvpa/lib/python3.8/site-packages/scipy/ndimage/measurements.py:305: DeprecationWarning: In future, it will be an error for 'np.bool_' scalars to be interpreted as an index
  return _nd_image.find_objects(input, max_label)

.................................................................................0.5 0.345679012345679
(0.5, 0.345679012345679)

RTFS_factor_1

r=-0.01; p-value=0.9434

rho=0.01; p-value=0.9354

RTFS_factor_2

r=0.17; p-value=0.2449

rho=0.13; p-value=0.3895

RTFS_f1_minus_f2

r=-0.17; p-value=0.2450

rho=-0.08; p-value=0.5956

SST_PostErrorSlowW1_median

r=0.14; p-value=0.3034

rho=0.20; p-value=0.1204

SST_PostErrorSlowW1_mean

r=0.13; p-value=0.3292

rho=0.22; p-value=0.0807

SST_pes_mean_limited

r=0.07; p-value=0.5884

rho=0.15; p-value=0.2424

FFQ_v2_Mean_Weighted_Nutrient_Density

r=-0.06; p-value=0.6069

rho=0.02; p-value=0.8459

(0.5, 0.345679012345679)

RTFS_factor_1

r=0.02; p-value=0.9091

rho=0.01; p-value=0.9667

RTFS_factor_2

r=-0.11; p-value=0.4522

rho=-0.07; p-value=0.6454

RTFS_f1_minus_f2

r=0.13; p-value=0.3832

rho=0.09; p-value=0.5507

SST_PostErrorSlowW1_median

r=-0.24; p-value=0.0707

rho=-0.28; p-value=0.0307

SST_PostErrorSlowW1_mean

r=-0.17; p-value=0.1842

rho=-0.26; p-value=0.0416

SST_pes_mean_limited

r=0.04; p-value=0.7810

rho=-0.01; p-value=0.9431

FFQ_v2_Mean_Weighted_Nutrient_Density

r=-0.09; p-value=0.4533

rho=-0.12; p-value=0.3068

pes_proba_masked = forced_choice_results_dict

def select_trials_by_type(df,trial_type):
    return(df[df['target_y']==trial_type])

def select_prob_col_with_label(df,prob_col):
    return(df.loc[:,['chunks',prob_col]])


# def select_CS_trials(df):
#     return(df[df['pred_y_forced_choice']=='CorrectStop'])

# def select_CS_and_labels(df):
#     return(df.loc[:,['chunks','pred_prob_CorrectStop']])

def label_cols(df, prepend):
    df1=df.copy()
    df1.columns = [prepend + c if c!="chunks" else c for c in df.columns]
    return(df1)

pes_proba_masked['harvardoxford-cortical_prob_Frontal Orbital Cortex']['sample_wise']

	chunks	target_y	pred_y	pred_y_forced_choice	pred_prob_CorrectGoFollowingCorrectStop	pred_prob_CorrectGoFollowingFailedStop	ln_inv_pred_prob_CGAfterCS	ln_inv_pred_prob_CGAfterFS
0	DEV005_1	CorrectGoFollowingCorrectStop	CorrectGoFollowingCorrectStop	CorrectGoFollowingCorrectStop	0.479206	0.520794	-0.652401	-0.735625
1	DEV005_1	CorrectGoFollowingFailedStop	CorrectGoFollowingCorrectStop	CorrectGoFollowingFailedStop	0.476081	0.523919	-0.646418	-0.742167
0	DEV006_1	CorrectGoFollowingCorrectStop	CorrectGoFollowingCorrectStop	CorrectGoFollowingCorrectStop	0.414901	0.585099	-0.535975	-0.879715
1	DEV006_1	CorrectGoFollowingFailedStop	CorrectGoFollowingCorrectStop	CorrectGoFollowingFailedStop	0.414843	0.585157	-0.535875	-0.879855
0	DEV010_1	CorrectGoFollowingCorrectStop	CorrectGoFollowingCorrectStop	CorrectGoFollowingCorrectStop	0.490349	0.509651	-0.674029	-0.712638
...	...	...	...	...	...	...	...	...
1	DEV216_1	CorrectGoFollowingFailedStop	CorrectGoFollowingCorrectStop	CorrectGoFollowingCorrectStop	0.482093	0.517907	-0.657960	-0.729618
0	DEV217_1	CorrectGoFollowingCorrectStop	CorrectGoFollowingCorrectStop	CorrectGoFollowingFailedStop	0.473232	0.526768	-0.640995	-0.748170
1	DEV217_1	CorrectGoFollowingFailedStop	CorrectGoFollowingCorrectStop	CorrectGoFollowingCorrectStop	0.476046	0.523954	-0.646351	-0.742241
0	DEV218_1	CorrectGoFollowingCorrectStop	CorrectGoFollowingCorrectStop	CorrectGoFollowingCorrectStop	0.467025	0.532975	-0.629281	-0.761372
1	DEV218_1	CorrectGoFollowingFailedStop	CorrectGoFollowingCorrectStop	CorrectGoFollowingFailedStop	0.458079	0.541921	-0.612636	-0.780713

162 rows × 8 columns

pes_proba_masked.keys()

dict_keys(['negative affect_association-test_z_FDR_0.01', 'emotion regulation_association-test_z_FDR_0.01', 'harvardoxford-cortical_prob_Frontal Medial Cortex', 'negative emotions_association-test_z_FDR_0.01', 'self referential_association-test_z_FDR_0.01', 'harvardoxford-cortical_prob_Frontal Orbital Cortex', 'harvardoxford-cortical_prob_Cingulate Gyrus, anterior division'])

CG_p_CS_wholebrain = label_cols(
    select_prob_col_with_label(
    select_trials_by_type(SVC_proba_wholebrain['sample_wise'],'CorrectGoFollowingCorrectStop'),
    "pred_prob_CorrectGoFollowingCorrectStop"
),'CG_p_CS_wholebrain_')

CG_p_FS_wholebrain = label_cols(
    select_prob_col_with_label(
    select_trials_by_type(SVC_proba_wholebrain['sample_wise'],'CorrectGoFollowingFailedStop'),
    "pred_prob_CorrectGoFollowingFailedStop"
),'CG_p_FS_wholebrain_')

CG_p_FS_emotion_regulation = label_cols(
    select_prob_col_with_label(
    select_trials_by_type(
        pes_proba_masked['emotion regulation_association-test_z_FDR_0.01']['sample_wise']
        ,'CorrectGoFollowingFailedStop'),
    "pred_prob_CorrectGoFollowingFailedStop"
),'CG_p_FS_Frontal Orbital Cortex_')

CG_p_FS_Frontal_Orbital_Cortex = label_cols(
    select_prob_col_with_label(
    select_trials_by_type(
        pes_proba_masked['harvardoxford-cortical_prob_Frontal Orbital Cortex']['sample_wise']
        ,'CorrectGoFollowingFailedStop'),
    "pred_prob_CorrectGoFollowingFailedStop"
),'CG_p_FS_Frontal Orbital Cortex_')

# CG_p_CS_ACC = label_cols(
#     select_prob_col_with_label(
#     select_trials_by_type(
#         pes_proba_masked['harvardoxford-cortical_prob_Cingulate Gyrus, anterior division']['sample_wise']
#         ,'CorrectGoFollowingCorrectStop'),
#     "pred_prob_CorrectGoFollowingCorrectStop"
# ),'CG_p_CS_ACC_')

# CG_p_CS_Negative_Affect = label_cols(
#     select_prob_col_with_label(
#     select_trials_by_type(
#         pes_proba_masked['negative affect_association-test_z_FDR_0.01']['sample_wise']
#         ,'CorrectGoFollowingCorrectStop'),
#     "pred_prob_CorrectGoFollowingCorrectStop"
# ),'CG_p_CS_negative affect_')

neural_all = pd.merge(pd.merge(pd.merge(
        CG_p_CS_wholebrain,
        CG_p_FS_wholebrain),
        CG_p_FS_emotion_regulation),
        CG_p_FS_Frontal_Orbital_Cortex)

neural_behavioral = pd.merge(
    neural_all,
    individual_differences,
    left_on='chunks',right_on='subj_wave')

neural_behavioral.to_csv(ml_data_folderpath + "/analysis_v5_pes_test1.csv")